

const https = require('https');
const cheerio = require('cheerio');
const fs = require('fs');
const baseUrl = 'https://www.airteamimages.com';
const maxPage = 284969; // 最大页码
const filePath = 'data/';
const request = require('request');

const MAX_RETRIES = 5; // 最大重试次数

let options = {
    host: 'www.airteamimages.com',
    path: '/boeing-737_N986NN_american-airlines_388536.html',
    method: 'GET',
    timeout: 5000, // 设置超时时间为5秒
    // port: 443,
    headers: {}
};

async function fetchData(page) {
    let retries = 0;

    while (retries < MAX_RETRIES) {
        try {
            return new Promise((resolve, reject) => {
                const req = https.request(options, (res) => {
                    let data = '';
                    res.setEncoding('utf-8');

                    res.on('data', (chunk) => {
                        data += chunk;
                    });

                    res.on('end', () => {
                        const images = [];

                        const $ = cheerio.load(data);
                        const imgSrc = $('#medimg-container-landscape img').attr('src');
                        const imageUrl = baseUrl + imgSrc;
                        images.push({ imageUrl });

                        const href = $('.imgnextprev .imgnext').attr('href');
                        options.path = href;
                        console.log(href, 'href');

                        $('.data-item').each((index, element) => {
                            const label = $(element).find('.label').text().replace(':', '').trim();
                            const value = $(element).find('.data').find('a').length > 0 ? $(element).find('.data').find('a').text().trim() : $(element).find('.data').text().trim();
                            let imgObj = new Object();
                            imgObj[label] = value;
                            images.push(imgObj);
                        });

                        resolve(images);
                    });
                });

                req.on('error', (e) => {
                    console.error(`请求遇到问题: ${e.message}`);
                    setTimeout(() => {
                        retries++;
                        resolve(fetchData(page));
                    }, 3000); // 等待3秒后重试
                });

                req.end();
            });
        } catch (error) {
            console.error(`请求失败，重试中 (${retries + 1}/${MAX_RETRIES})`);
            retries++;
        }
    }

    throw new Error(`请求失败达到最大重试次数 (${MAX_RETRIES})`);
}

async function scrapeData() {
    let i = 36636;

    async function crawl() {
        if (i > maxPage) {
            return;
        }

        try {
            const dataArray = await fetchData(i);

            console.log(`爬取到第${i}数据`);

            const jsonData = JSON.stringify(dataArray);
            let image_url = (dataArray[0].imageUrl.split('pics/')[1].split('/')[1].split('_')[0]).toUpperCase();
            let filename = filePath + image_url.split("").join('/') + '/' + image_url + '.json';
            let imagename = filePath + image_url.split("").join('/') + '/' + image_url + '.' + dataArray[0].imageUrl.split('_')[1].split('.')[1];
            let file_path = filePath + image_url.split("").join('/');

            if (!fs.existsSync(file_path)) {
                fs.mkdirSync(file_path, { recursive: true });
            }

            await new Promise((resolve, reject) => {
                request(dataArray[0].imageUrl).pipe(fs.createWriteStream(imagename)).on('close', function() {
                    console.log('图片保存完成');
                    resolve();
                });
            });

            await new Promise((resolve, reject) => {
                fs.writeFile(filename, jsonData, (err) => {
                    if (err) {
                        console.error(`写入文件 ${filename} 出错：`, err);
                        reject(err);
                    } else {
                        console.log(`数据已写入 ${filename}`);
                        resolve();
                    }
                });
            });

        } catch (error) {
            console.error('失败',error);
        }

        i++;
        setTimeout(crawl, 1000); // 每秒钟执行一次 crawl 函数
    }

    await crawl();
}

scrapeData();

